//==============================================================================
//
//             DLexer - the lexer class in the ofc-library
//
//               Copyright (C) 2003  Dick van Oudheusden
//  
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Public License for more details.
//
// You should have received a copy of the GNU General Public
// License along with this library; if not, write to the Free
// Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
//
//==============================================================================
// 
//  $Date: 2004-05-15 20:09:03 $ $Revision: 1.2 $
//
//==============================================================================

#include "ofc/DTokenizer.h"

#include <ctype.h>


#if _INTERFACE_

#include "ofc/config.h"

#include "ofc/DSource.h"

#include "ofc/DText.h"
#include "ofc/DList.h"

#include "ofc/DTextReadable.h"


#define DTK_UNKNOWN     (0)
#define DTK_EOFF        (1)
#define DTK_EXTRA1      (2)
#define DTK_EXTRA2      (3)
#define DTK_EXTRA3      (4)
#define DTK_WHITESPACE  (5)
#define DTK_COMMENT     (6)
#define DTK_NUMBER      (7)
#define DTK_KEYWORD     (8)
#define DTK_STRING      (9)
#define DTK_OPERATOR    (10)


// The class DTokenizer implements a (simple) lexical scanner. The 
// tokenizer scans a text stream for whitespace, strings, keywords, 
// comments, operators, numbers and three to be implemented formats. The 
// scanner scans the text on a line basis.
// The tokenizer does not use regular expressions, but it uses for every token
// a method to scan for the token. The method can be overridden for a specific
// implementation of the token scanner.
// There are two methods: nextToken and checkToken. The first method checks
// for a token and if there is a match, the scanner location is moved to
// the next location in the source. The second method only returns 
// the result of the match, the client must call the service
// 'next' to move the scanner location.
// The token methods return one of the following: DTK_UNKNOWN, DTK_EOFF,
// DTK_EXTRA1, DTK_EXTRA2, DTK_EXTRA3, DTK_WHITESPACE, DTK_COMMENT, 
// DTK_NUMBER, DTK_KEYWORD, DTK_STRING, DTK_OPERATOR.

@interface DTokenizer : Object
{
@private
  DList       *_sources;    // the stack with the sources
  DSource     *_source;     // the source for the tokenizer
   
  DText       *_text;       // the last scanned text
  BOOL         _eoff;       // is end of file reached ?
  unsigned     _scanned;    // the length of the scanned text
  BOOL         _skipWhiteSpace; // should whitespace be skipped ? (def. YES)
}

#endif



@implementation DTokenizer


//// Constructors

//
// Initialise a tokenizer
//
// @return the object
//

- (DTokenizer *) init
{
  [super init];

  _sources        = [DList new];
  _source         = nil;
   
  _text           = nil;
  _eoff           = NO;
  _scanned        = 0;
  _skipWhiteSpace = YES;

  return self;
}

//
// Initialise a tokenizer with a file
//
// @param source   the source
// @param name     the name for the source
// 
// @return the object
//

- (DTokenizer *) init :(id <DTextReadable>) source :(const char *) name
{
  [self init];
  
  [self source :source :name];
  
  return self;
}

//// Copy related methods

// 
// Deepen a copy of the object (not implemented)
//
// @return the object
//

- deepen
{
  [super deepen];

  WARNING(DW_METHOD_NOT_IMPL, "deepen");

  return self;
}


//// Deconstructor

//
// Free the Tokenizer (but *NOT* the TextReadables)
//
// @return the object
//

- free
{
  [_sources free];

  if (_text != nil)
    [_text free];

  if (_source != nil)
    [_source free];

  return [super free];
}


//// Member methods

//
// Get the last scanned text
// 
// @return the last scanned text (or NULL)
// 

- (const char *) text
{
  return ([_text cstring]);
}

//
// Get the name of the current file
// 
// @return the name of the file in a new DText (or nil)
// 

- (const char *) name
{
  return ((_source != nil) ? [_source name] : NULL);
}
   
//
// Get the current line number
//
// @return the current line number
//

- (int) lineNumber
{
  return ((_source != nil) ? [_source lineNumber] : 0);
}

//
// Get the current column number
//
// @return the current column number
//

- (int) columnNumber
{
  return ((_source != nil) ? [_source columnNumber] : 0);
}

//
// Is the end of file reached ?
// 
// @return is it ?
// 

- (BOOL) isEof
{
  return _eoff;
}

//
// Return whether white space is be skipped 
// 
// @return is it ?
// 

- (BOOL) skipWhiteSpace
{
  return _skipWhiteSpace;
}

//
// Set whether white space should be skipped
// 
// @param skip          should white space be skipped
// 
// @return the object
// 

- (DTokenizer *) skipWhiteSpace :(BOOL) skip
{
  _skipWhiteSpace = skip;
  
  return self;
}


//// Basic scanner methods (to be overridden)

//
// Check for white space (based on isspace)
// 
// @param text          the text to be scanned
// 
// @return the scanned text (or nil)
// 

- (DText *) whiteSpace :(const char *) text 
{
  DText *scanned = nil;
  
  if (isspace(*text))
  {
    scanned = [DText new];
    
    do
    {
      [scanned push :*text];
      
      text++;
    }
    while (isspace(*text));
  }
  
  return scanned;
}

//
// Check for an comment (start with '#' till eofl)
// 
// @param text          the text to be scanned
// 
// @return the scanned text (or nil)
// 

- (DText *) comment :(const char *) text 
{
  DText *scanned = nil;
  
  if (*text == '#')
  {
    scanned = [DText new];
    
    while (*text != EOS)
    {
      [scanned push :*text++];
    }
  }
  return scanned;
}

//
// Check for a keyword ([A-Za-z_][A-Za-z0-9_]*)
// 
// @param text          the text to be scanned
// 
// @return the scanned text (or nil)
// 

- (DText *) keyword :(const char *) text 
{
  DText *scanned = nil;
  
  if ((isalpha(*text)) || (*text == '_'))
  {
    scanned = [DText new];
    
    while ((isalnum(*text)) || (*text == '_'))
    {
      [scanned push :*text++];
    }
  }
  return scanned;
}

//
// Check for a string (".." or '..', \" and \' allowed)
// 
// @param text          the text to be scanned
// 
// @return the scanned text (or nil)
// 

- (DText *) string :(const char *) text 
{
  DText *scanned = nil;
  
  if ((*text == '"') || (*text == '\''))
  {
    char ch = *text;
    
    scanned = [DText new];
    
    [scanned push :*text++];
    
    while ((*text != ch) && (*text != EOS))
    {
      [scanned push :*text];
      
      if (*text == '\\')
      {
        text++;
        [scanned push :*text];
      }
      
      text++;
    }
    
    if (*text == ch)
      [scanned push :*text];
  }
  
  return scanned;
}

//
// Check for an operator (ispunct)
// 
// @param text          the text to be scanned
// 
// @return the scanned text (or nil)
// 

- (DText *) operator :(const char *) text
{
  DText *scanned = nil;

  if ((ispunct(*text)) && (*text != '"') && (*text != '\''))
  {
    scanned = [DText new];
     
    while ((ispunct(*text)) && (*text != '"') && (*text != '\''))
    {
      [scanned push :*text++];
    }
  }
  
  return scanned;
}

//
// Check for a number (isdigit)
// 
// @param text          the text to be scanned
// 
// @return the scanned text (or nil)
// 

- (DText *) number :(const char *) text
{
  DText *scanned = nil;

  if (isdigit(*text))
  {
    scanned = [DText new];
    
    while (isdigit(*text))
    {
      [scanned push :*text++];
    }
  }
  
  return scanned;
}

//
// Check for extra token 1 (to be overridden)
// 
// @param text          the text to be scanned
// 
// @return the scanned text (or nil)
// 

- (DText *) extraToken1 :(const char *) text
{
  return nil;
}

//
// Check for extra token 2 (to be overridden)
// 
// @param text          the text to be scanned
// 
// @return the scanned text (or nil)
// 

- (DText *) extraToken2 :(const char *) text 
{
  return nil;
}

//
// Check for extra token 3 (to be overridden)
// 
// @param text          the text to be scanned
// 
// @return the scanned text (or nil)
// 

- (DText *) extraToken3 :(const char *) text 
{
  return nil;
}


//// Source methods

//
// Start using a source (an already open source is pushed on the sources stack)
// 
// @param source   the source
// @param name     the name for the source
// 

- (BOOL) source :(id <DTextReadable>) source :(const char *) name
{
  if (_source != nil)
  {
    [_sources append :source];
  }

  _source = [DSource new];
  
  return [_source set :source :name];
}

//
// Pop a source from the sources stack
// 
// @return is a source popped from the sources stack ?
// 

- (BOOL) popSource
{
  BOOL ok = NO;
  
  if ([_sources length] > 0)
  {
    if (_source != nil)
      [_source free];
    
    _source = [_sources pop];
    
    ok = YES;
  }
  
  return ok;
}


//// Scanner methods

//
// Scan the source for the next token (white space is skipped if
// this is active)
// 
// @return the next token
// 

- (int) nextToken 
{
  int token;
  
  do
  {
    token = [self checkToken];
  
    if (token != DTK_EOFF)
    {
      [self next];
    }
  }
  while ((token == DTK_WHITESPACE) && (_skipWhiteSpace));

  return token;
}

//
// Check the source for the next token (white space is not skipped)
//
// @return the found token
// 

- (int) checkToken
{
  int token = DTK_EOFF;

  _scanned = 0;
  
  if (_text != nil)
  {
    [_text free];
    
    _text = nil;
  }
  
  if ((_source != nil) && (!_eoff))
  {
    const char *line = [_source line];
    
    if ((_text = [self extraToken1 :line]) != nil)
    {
      token = DTK_EXTRA1;
    }
    else if ((_text = [self extraToken2 :line]) != nil)
    {
      token = DTK_EXTRA2;
    }
    else if ((_text = [self extraToken3 :line]) != nil)
    {
      token = DTK_EXTRA3;
    }
    else if ((_text = [self whiteSpace :line]) != nil)
    {
      token = DTK_WHITESPACE;
    }
    else if ((_text = [self comment :line]) != nil)
    {
      token = DTK_COMMENT;
    }
    else if ((_text = [self number :line]) != nil)
    {
      token = DTK_NUMBER;
    }
    else if ((_text = [self keyword :line]) != nil)
    {
      token = DTK_KEYWORD;
    }
    else if ((_text = [self string :line]) != nil)
    {
      token = DTK_STRING;
    }
    else if ((_text = [self operator :line]) != nil)
    {
      token = DTK_OPERATOR;
    }
    else
    {
      token = DTK_UNKNOWN;
      
      _text = [DText new];
      
      [_text push :line[0]];
    }
    
    if (_text != nil)
    {
      _scanned = [_text length];
    }
  }
  
  return (token);
}

//
// Move the scanned indicator after a checkToken method
// 
// @return the object
// 

- (DTokenizer *) next 
{
  if (_scanned > 0)
  {
    [self next :_scanned];
    
    _scanned = 0;
  }

  return self;
}

//
// Move the scanned indicator a number of positions (private)
//
// @param positions  the number of position to move
//  
// @return the object
// 

- (DTokenizer *) next :(unsigned) positions
{
  if (_source != nil)
  {
    if ([_source scanned :positions])
    {
      do
      {
        if (![_source nextLine])
        {
          if (![self popSource])
          {
            _eoff = YES;
          }
        }
      }
      while ((!_eoff) && ([_source isLineScanned]));
    }
  }
  
  return self;
}
    

//// Error methods

//
// Generate an error on stderr for the current file and line
// 
// @param msg      the error message
// 
// @return the object
// 

- (DTokenizer *) error :(const char *) msg
{
  if (_source == nil)
  {
    WARNING(DW_OBJECT_NOT_INIT, "source");
  }
  else
  {
    [_source error :msg];
  }
  
  return self;
}
          
@end

/*==========================================================================*/
